{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.导入库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.layers import Dense, Conv2D, BatchNormalization, Activation\n",
    "from keras.layers import AveragePooling2D, Input, Flatten\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.models import Model\n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model parameter\n",
    "# ----------------------------------------------------------------------------\n",
    "#           |      | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch\n",
    "# Model     |  n   | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti\n",
    "#           |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)\n",
    "# ----------------------------------------------------------------------------\n",
    "# ResNet20  | 3 (2)| 92.16     | 91.25     | -----     | -----     | 35 (---)\n",
    "# ResNet32  | 5(NA)| 92.46     | 92.49     | NA        | NA        | 50 ( NA)\n",
    "# ResNet44  | 7(NA)| 92.50     | 92.83     | NA        | NA        | 70 ( NA)\n",
    "# ResNet56  | 9 (6)| 92.71     | 93.03     | 93.01     | NA        | 90 (100)\n",
    "# ResNet110 |18(12)| 92.65     | 93.39+-.16| 93.15     | 93.63     | 165(180)\n",
    "# ResNet164 |27(18)| -----     | 94.07     | -----     | 94.54     | ---(---)\n",
    "# ResNet1001| (111)| -----     | 92.39     | -----     | 95.08+-.14| ---(---)\n",
    "# ---------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 定义函数load_train_dataset、load_test_dataset，用于加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets.cifar import load_batch\n",
    "\n",
    "# 加载数据集cifar10里面的训练集\n",
    "def load_train_dataset(dirPath='../resources/cifar-10-batches-py/'):\n",
    "    train_sample_quantity = 50000\n",
    "    image_width = 32\n",
    "    image_height = 32\n",
    "    channel_quantity = 3\n",
    "    train_x = np.zeros((train_sample_quantity, channel_quantity, image_width, image_height),\n",
    "                       dtype='uint8')\n",
    "    train_y = np.zeros((train_sample_quantity, ),\n",
    "                       dtype='uint8')\n",
    "    for i in range(1, 6):\n",
    "        fileName = 'data_batch_%d' %i\n",
    "        filePath = os.path.join(dirPath, fileName)\n",
    "        startIndex = (i - 1) * 10000\n",
    "        endIndex = i * 10000\n",
    "        train_x[startIndex:endIndex, :, :, :], train_y[startIndex:endIndex] = load_batch(filePath)\n",
    "    print('未做矩阵转置：', train_x.shape)\n",
    "    # 从官网上下载的数据集的4个维度为样本个数n、通道数c、宽度w、高度h\n",
    "    # Keras基于Tensorflow，数据的维度顺序要求：样本个数n、宽度w、高度h、通道数c，所以使用np.transpose完成矩阵转置\n",
    "    train_x = train_x.transpose(0, 2, 3, 1)\n",
    "    print('矩阵转置后：', train_x.shape)\n",
    "    return train_x, train_y\n",
    "\n",
    "# 加载数据集cifar10里面的测试集\n",
    "def load_test_dataset(dirPath='../resources/cifar-10-batches-py/'):\n",
    "    fileName = 'test_batch'\n",
    "    filePath = os.path.join(dirPath, fileName)\n",
    "    test_x, test_y = load_batch(filePath)\n",
    "    print('未做矩阵转置：', test_x.shape)\n",
    "    test_x = test_x.transpose(0, 2, 3, 1)\n",
    "    print('矩阵转置后：', test_x.shape)\n",
    "    return test_x, test_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 对加载后的图像数据做处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "未做矩阵转置： (50000, 3, 32, 32)\n",
      "矩阵转置后： (50000, 32, 32, 3)\n",
      "未做矩阵转置： (10000, 3, 32, 32)\n",
      "矩阵转置后： (10000, 32, 32, 3)\n",
      "x_train shape: (50000, 32, 32, 3)\n",
      "y_train shape: (50000,)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "未做to_categorical之前, y_train shape: (50000,)\n",
      "做to_categorical之后, y_train shape: (50000, 10)\n"
     ]
    }
   ],
   "source": [
    "# Subtracting pixel mean improves accuracy\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "# Load the CIFAR10 data.\n",
    "dirPath = '../resources/cifar-10-batches-py/'\n",
    "x_train, y_train = load_train_dataset(dirPath)\n",
    "x_test, y_test = load_test_dataset(dirPath)\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "num_classes = 10\n",
    "print('未做to_categorical之前, y_train shape:', y_train.shape)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "print('做to_categorical之后, y_train shape:', y_train.shape)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.搭建神经网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 定义函数resnet_layer，返回值是经过resnet_layer计算的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n",
    "    # Arguments\n",
    "        inputs (tensor): input tensor from input image or previous layer\n",
    "        num_filters (int): Conv2D number of filters\n",
    "        kernel_size (int): Conv2D square kernel dimensions\n",
    "        strides (int): Conv2D square stride dimensions\n",
    "        activation (string): activation name\n",
    "        batch_normalization (bool): whether to include batch normalization\n",
    "        conv_first (bool): conv-bn-activation (True) or\n",
    "            bn-activation-conv (False)\n",
    "    # Returns\n",
    "        x (tensor): tensor as input to the next layer\n",
    "    \"\"\"\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=l2(1e-4))\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 定义函数resnet_v1，返回值是模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v1(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 1 Model builder [a]\n",
    "    Stacks of 2 x (3 x 3) Conv2D-BN-ReLU\n",
    "    Last ReLU is after the shortcut connection.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filters is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same number of filters.\n",
    "    Features maps sizes:\n",
    "    stage 0: 32x32, 16\n",
    "    stage 1: 16x16, 32\n",
    "    stage 2:  8x8,  64\n",
    "    The Number of parameters is approx the same as Table 6 of [a]:\n",
    "    ResNet20 0.27M\n",
    "    ResNet32 0.46M\n",
    "    ResNet44 0.66M\n",
    "    ResNet56 0.85M\n",
    "    ResNet110 1.7M\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 6 != 0:\n",
    "        raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')\n",
    "    # Start model definition.\n",
    "    num_filters = 16\n",
    "    num_res_blocks = int((depth - 2) / 6)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = resnet_layer(inputs=inputs)\n",
    "    # Instantiate the stack of residual units\n",
    "    for stack in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            strides = 1\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                strides = 2  # downsample\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters,\n",
    "                             strides=strides)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters,\n",
    "                             activation=None)\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "            x = Activation('relu')(x)\n",
    "        num_filters *= 2\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v1 does not use BN after last shortcut connection-ReLU\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 定义函数resnet_v2，返回值是模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v2(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 2 Model builder [b]\n",
    "    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n",
    "    bottleneck layer\n",
    "    First shortcut connection per layer is 1 x 1 Conv2D.\n",
    "    Second and onwards shortcut connection is identity.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filter maps is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same filter map sizes.\n",
    "    Features maps sizes:\n",
    "    conv1  : 32x32,  16\n",
    "    stage 0: 32x32,  64\n",
    "    stage 1: 16x16, 128\n",
    "    stage 2:  8x8,  256\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = True\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 实例化模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 2\n",
    "\n",
    "# Computed depth from supplied model parameter n\n",
    "n = 2\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "    \n",
    "# 根据ResNet版本，获取对应的模型对象\n",
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.001),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.5 打印模型架构信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet20v2\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   272         activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 16)   2320        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 64)   1088        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 64)   1088        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 64)   0           conv2d_5[0][0]                   \n",
      "                                                                 conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 64)   256         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 64)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   1040        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 32, 32, 16)   2320        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 16)   64          conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 16)   0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 32, 32, 64)   1088        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 32, 32, 64)   0           add_1[0][0]                      \n",
      "                                                                 conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 32, 32, 64)   256         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 32, 32, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 16, 16, 64)   4160        activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 16, 16, 64)   256         conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 16, 16, 64)   0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 16, 16, 64)   36928       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 16, 16, 64)   256         conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 16, 16, 64)   0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 16, 16, 128)  8320        add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 16, 16, 128)  8320        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 16, 16, 128)  0           conv2d_12[0][0]                  \n",
      "                                                                 conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 16, 16, 128)  512         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 16, 16, 128)  0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 16, 16, 64)   8256        activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 16, 16, 64)   256         conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 16, 16, 64)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 16, 16, 64)   36928       activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 16, 16, 64)   256         conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 16, 16, 64)   0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 16, 16, 128)  8320        activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 16, 16, 128)  0           add_3[0][0]                      \n",
      "                                                                 conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 16, 16, 128)  512         add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 16, 16, 128)  0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 8, 8, 128)    16512       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 8, 8, 128)    512         conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 8, 8, 128)    0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 8, 8, 128)    147584      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 8, 8, 128)    512         conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 8, 8, 128)    0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 8, 8, 256)    33024       add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 8, 8, 256)    33024       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 8, 8, 256)    0           conv2d_19[0][0]                  \n",
      "                                                                 conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 8, 8, 256)    1024        add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 8, 8, 256)    0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 8, 8, 128)    32896       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 8, 8, 128)    512         conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 8, 8, 128)    0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 8, 8, 128)    147584      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 8, 8, 128)    512         conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 8, 8, 128)    0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 8, 8, 256)    33024       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 8, 8, 256)    0           add_5[0][0]                      \n",
      "                                                                 conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 8, 8, 256)    1024        add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 8, 8, 256)    0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 1, 1, 256)    0           activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           2570        flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 574,090\n",
      "Trainable params: 570,602\n",
      "Non-trainable params: 3,488\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model name, depth and version\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "print(model_type)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 规划学习率，训练到后期，学习率需要减少"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 模型训练时的参数设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training parameters\n",
    "batch_size = 64  # orig paper trained all networks with batch_size=128\n",
    "epochs = 200\n",
    "\n",
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.abspath('../resources/saved_models')\n",
    "model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=0,\n",
    "                             save_best_only=True)\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 使用图像增强的结果做模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n",
      "Epoch 1/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 43s 55ms/step - loss: 1.8109 - acc: 0.4717 - val_loss: 1.7204 - val_acc: 0.4875\n",
      "Epoch 2/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 1.3825 - acc: 0.6124 - val_loss: 1.6215 - val_acc: 0.5578\n",
      "Epoch 3/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 1.2082 - acc: 0.6676 - val_loss: 1.2750 - val_acc: 0.6423\n",
      "Epoch 4/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 1.0799 - acc: 0.7093 - val_loss: 1.4687 - val_acc: 0.6071\n",
      "Epoch 5/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.9951 - acc: 0.7393 - val_loss: 1.2271 - val_acc: 0.6873\n",
      "Epoch 6/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.9326 - acc: 0.7593 - val_loss: 1.5300 - val_acc: 0.6386\n",
      "Epoch 7/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.8875 - acc: 0.7737 - val_loss: 1.0115 - val_acc: 0.7413\n",
      "Epoch 8/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.8500 - acc: 0.7842 - val_loss: 1.0557 - val_acc: 0.7304\n",
      "Epoch 9/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.8163 - acc: 0.7967 - val_loss: 1.0847 - val_acc: 0.7097\n",
      "Epoch 10/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.7907 - acc: 0.8050 - val_loss: 1.1323 - val_acc: 0.7173\n",
      "Epoch 11/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.7708 - acc: 0.8096 - val_loss: 0.9245 - val_acc: 0.7627\n",
      "Epoch 12/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.7455 - acc: 0.8190 - val_loss: 1.1161 - val_acc: 0.7185\n",
      "Epoch 13/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.7311 - acc: 0.8234 - val_loss: 0.9749 - val_acc: 0.7556\n",
      "Epoch 14/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.7121 - acc: 0.8308 - val_loss: 0.8557 - val_acc: 0.7916\n",
      "Epoch 15/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.6997 - acc: 0.8324 - val_loss: 0.9865 - val_acc: 0.7414\n",
      "Epoch 16/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.6841 - acc: 0.8383 - val_loss: 0.8175 - val_acc: 0.7997\n",
      "Epoch 17/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.6710 - acc: 0.8417 - val_loss: 1.0153 - val_acc: 0.7592\n",
      "Epoch 18/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.6642 - acc: 0.8431 - val_loss: 0.8401 - val_acc: 0.7889\n",
      "Epoch 19/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.6524 - acc: 0.8486 - val_loss: 0.9217 - val_acc: 0.7728\n",
      "Epoch 20/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.6403 - acc: 0.8530 - val_loss: 0.8890 - val_acc: 0.7835\n",
      "Epoch 21/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.6293 - acc: 0.8550 - val_loss: 0.8319 - val_acc: 0.7993\n",
      "Epoch 22/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.6227 - acc: 0.8578 - val_loss: 0.8520 - val_acc: 0.7915\n",
      "Epoch 23/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.6173 - acc: 0.8590 - val_loss: 0.8887 - val_acc: 0.7845\n",
      "Epoch 24/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.6093 - acc: 0.8610 - val_loss: 0.8801 - val_acc: 0.7922\n",
      "Epoch 25/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.6014 - acc: 0.8659 - val_loss: 0.8482 - val_acc: 0.7943\n",
      "Epoch 26/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5923 - acc: 0.8676 - val_loss: 0.8420 - val_acc: 0.7919\n",
      "Epoch 27/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5892 - acc: 0.8677 - val_loss: 0.8518 - val_acc: 0.7948\n",
      "Epoch 28/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5846 - acc: 0.8723 - val_loss: 0.7608 - val_acc: 0.8193\n",
      "Epoch 29/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5810 - acc: 0.8713 - val_loss: 0.9059 - val_acc: 0.7984\n",
      "Epoch 30/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5752 - acc: 0.8722 - val_loss: 0.8059 - val_acc: 0.8085\n",
      "Epoch 31/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.5658 - acc: 0.8769 - val_loss: 0.6638 - val_acc: 0.8469\n",
      "Epoch 32/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5636 - acc: 0.8764 - val_loss: 0.8879 - val_acc: 0.7879\n",
      "Epoch 33/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.5584 - acc: 0.8789 - val_loss: 0.7955 - val_acc: 0.8119\n",
      "Epoch 34/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5543 - acc: 0.8800 - val_loss: 0.9809 - val_acc: 0.7814\n",
      "Epoch 35/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5514 - acc: 0.8823 - val_loss: 0.7013 - val_acc: 0.8372\n",
      "Epoch 36/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5480 - acc: 0.8812 - val_loss: 0.7925 - val_acc: 0.8157\n",
      "Epoch 37/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5400 - acc: 0.8862 - val_loss: 0.7561 - val_acc: 0.8244\n",
      "Epoch 38/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.5400 - acc: 0.8859 - val_loss: 0.7081 - val_acc: 0.8344\n",
      "Epoch 39/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5341 - acc: 0.8874 - val_loss: 0.7528 - val_acc: 0.8271\n",
      "Epoch 40/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5304 - acc: 0.8883 - val_loss: 0.7722 - val_acc: 0.8273\n",
      "Epoch 41/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5305 - acc: 0.8875 - val_loss: 0.7210 - val_acc: 0.8339\n",
      "Epoch 42/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5288 - acc: 0.8877 - val_loss: 0.7504 - val_acc: 0.8221\n",
      "Epoch 43/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5265 - acc: 0.8887 - val_loss: 0.6577 - val_acc: 0.8541\n",
      "Epoch 44/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 45ms/step - loss: 0.5192 - acc: 0.8929 - val_loss: 0.8218 - val_acc: 0.8178\n",
      "Epoch 45/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5214 - acc: 0.8908 - val_loss: 0.7735 - val_acc: 0.8195\n",
      "Epoch 46/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5137 - acc: 0.8950 - val_loss: 0.8053 - val_acc: 0.8238\n",
      "Epoch 47/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5128 - acc: 0.8936 - val_loss: 0.7658 - val_acc: 0.8198\n",
      "Epoch 48/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.5155 - acc: 0.8923 - val_loss: 0.7398 - val_acc: 0.8275\n",
      "Epoch 49/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5052 - acc: 0.8959 - val_loss: 0.6959 - val_acc: 0.8421\n",
      "Epoch 50/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5078 - acc: 0.8945 - val_loss: 0.7328 - val_acc: 0.8291\n",
      "Epoch 51/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5043 - acc: 0.8977 - val_loss: 0.7169 - val_acc: 0.8373\n",
      "Epoch 52/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4977 - acc: 0.8987 - val_loss: 0.8224 - val_acc: 0.8191\n",
      "Epoch 53/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.5000 - acc: 0.8981 - val_loss: 0.7904 - val_acc: 0.8134\n",
      "Epoch 54/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.4965 - acc: 0.8997 - val_loss: 0.7446 - val_acc: 0.8333\n",
      "Epoch 55/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4901 - acc: 0.9022 - val_loss: 0.6892 - val_acc: 0.8453\n",
      "Epoch 56/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 45ms/step - loss: 0.4986 - acc: 0.8984 - val_loss: 0.7475 - val_acc: 0.8160\n",
      "Epoch 57/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4915 - acc: 0.9002 - val_loss: 0.6414 - val_acc: 0.8609\n",
      "Epoch 58/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4878 - acc: 0.9018 - val_loss: 0.8573 - val_acc: 0.8049\n",
      "Epoch 59/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.4895 - acc: 0.9008 - val_loss: 0.8394 - val_acc: 0.8087\n",
      "Epoch 60/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.4877 - acc: 0.9021 - val_loss: 0.6497 - val_acc: 0.8595\n",
      "Epoch 61/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4867 - acc: 0.9032 - val_loss: 0.8252 - val_acc: 0.8059\n",
      "Epoch 62/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.4820 - acc: 0.9022 - val_loss: 0.6978 - val_acc: 0.8501\n",
      "Epoch 63/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.4803 - acc: 0.9036 - val_loss: 0.8051 - val_acc: 0.8203\n",
      "Epoch 64/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4768 - acc: 0.9056 - val_loss: 0.7792 - val_acc: 0.8371\n",
      "Epoch 65/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 37s 47ms/step - loss: 0.4794 - acc: 0.9051 - val_loss: 0.6775 - val_acc: 0.8516\n",
      "Epoch 66/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.4741 - acc: 0.9063 - val_loss: 0.7343 - val_acc: 0.8365\n",
      "Epoch 67/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4741 - acc: 0.9070 - val_loss: 0.9716 - val_acc: 0.7827\n",
      "Epoch 68/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.4768 - acc: 0.9049 - val_loss: 0.6715 - val_acc: 0.8469\n",
      "Epoch 69/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 46ms/step - loss: 0.4696 - acc: 0.9082 - val_loss: 0.7315 - val_acc: 0.8324\n",
      "Epoch 70/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.4682 - acc: 0.9081 - val_loss: 0.6989 - val_acc: 0.8506\n",
      "Epoch 71/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.4673 - acc: 0.9087 - val_loss: 0.6554 - val_acc: 0.8557\n",
      "Epoch 72/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 34s 44ms/step - loss: 0.4679 - acc: 0.9086 - val_loss: 0.7586 - val_acc: 0.8362\n",
      "Epoch 73/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 36s 45ms/step - loss: 0.4622 - acc: 0.9105 - val_loss: 0.6737 - val_acc: 0.8510\n",
      "Epoch 74/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 45ms/step - loss: 0.4684 - acc: 0.9080 - val_loss: 0.6612 - val_acc: 0.8612\n",
      "Epoch 75/200\n",
      "Learning rate:  0.001\n",
      "782/782 [==============================] - 35s 44ms/step - loss: 0.4658 - acc: 0.9102 - val_loss: 0.7160 - val_acc: 0.8432\n",
      "Epoch 76/200\n",
      "Learning rate:  0.001\n",
      "742/782 [===========================>..] - ETA: 5s - loss: 0.4639 - acc: 0.9090"
     ]
    }
   ],
   "source": [
    "data_augmentation = True\n",
    "if data_augmentation:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs,\n",
    "                        verbose=1, \n",
    "                        workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.模型评估"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 加载训练好的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "加载最后一个epoch保存的模型文件，路径：../resources/saved_models/cifar10_ResNet20v2_model.124.h5\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "def load_best(dirPath, partOfFileName):\n",
    "    fileName_list = next(os.walk(dirPath))[2]\n",
    "    part_fileName_list = [k for k in fileName_list if partOfFileName in k]\n",
    "    assert len(part_fileName_list), 'wrong argument: partOfFileName, please check saved model directory' \n",
    "    part_fileName_list_1 = sorted(part_fileName_list, key=lambda x: int(x.split('.')[1]))\n",
    "    best_fileName = part_fileName_list_1[-1]\n",
    "    best_filePath = os.path.join(dirPath, best_fileName)\n",
    "    model = load_model(best_filePath)\n",
    "    print('加载最后一个epoch保存的模型文件，路径：%s' %best_filePath)\n",
    "    return model\n",
    "\n",
    "version = 2\n",
    "# 设置n为6，可以加载ResNet56v2模型，在cifar10数据集准确率为93.29%\n",
    "n = 2\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "partOfFileName = '%s_model' %model_type\n",
    "dirPath = '../resources/saved_models/'\n",
    "model = load_best(dirPath, partOfFileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 计算分类模型的准确率 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.001),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 8s 828us/step\n",
      "Test loss: 0.42392853331565855\n",
      "Test accuracy: 0.914\n"
     ]
    }
   ],
   "source": [
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
